{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import math\n",
    "import numpy as np\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>ps_ind_01</th>\n",
       "      <th>ps_ind_02_cat</th>\n",
       "      <th>ps_ind_03</th>\n",
       "      <th>ps_ind_04_cat</th>\n",
       "      <th>ps_ind_05_cat</th>\n",
       "      <th>ps_ind_06_bin</th>\n",
       "      <th>ps_ind_07_bin</th>\n",
       "      <th>ps_ind_08_bin</th>\n",
       "      <th>...</th>\n",
       "      <th>ps_calc_11</th>\n",
       "      <th>ps_calc_12</th>\n",
       "      <th>ps_calc_13</th>\n",
       "      <th>ps_calc_14</th>\n",
       "      <th>ps_calc_15_bin</th>\n",
       "      <th>ps_calc_16_bin</th>\n",
       "      <th>ps_calc_17_bin</th>\n",
       "      <th>ps_calc_18_bin</th>\n",
       "      <th>ps_calc_19_bin</th>\n",
       "      <th>ps_calc_20_bin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 59 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  target  ps_ind_01  ps_ind_02_cat  ps_ind_03  ps_ind_04_cat  \\\n",
       "0   7       0          2              2          5              1   \n",
       "1   9       0          1              1          7              0   \n",
       "2  13       0          5              4          9              1   \n",
       "3  16       0          0              1          2              0   \n",
       "4  17       0          0              2          0              1   \n",
       "\n",
       "   ps_ind_05_cat  ps_ind_06_bin  ps_ind_07_bin  ps_ind_08_bin       ...        \\\n",
       "0              0              0              1              0       ...         \n",
       "1              0              0              0              1       ...         \n",
       "2              0              0              0              1       ...         \n",
       "3              0              1              0              0       ...         \n",
       "4              0              1              0              0       ...         \n",
       "\n",
       "   ps_calc_11  ps_calc_12  ps_calc_13  ps_calc_14  ps_calc_15_bin  \\\n",
       "0           9           1           5           8               0   \n",
       "1           3           1           1           9               0   \n",
       "2           4           2           7           7               0   \n",
       "3           2           2           4           9               0   \n",
       "4           3           1           1           3               0   \n",
       "\n",
       "   ps_calc_16_bin  ps_calc_17_bin  ps_calc_18_bin  ps_calc_19_bin  \\\n",
       "0               1               1               0               0   \n",
       "1               1               1               0               1   \n",
       "2               1               1               0               1   \n",
       "3               0               0               0               0   \n",
       "4               0               0               1               1   \n",
       "\n",
       "   ps_calc_20_bin  \n",
       "0               1  \n",
       "1               0  \n",
       "2               0  \n",
       "3               0  \n",
       "4               0  \n",
       "\n",
       "[5 rows x 59 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('../kaggle_porto_seguro/train.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 21694 entries, 9 to 595158\n",
      "Data columns (total 59 columns):\n",
      "id                21694 non-null int64\n",
      "target            21694 non-null int64\n",
      "ps_ind_01         21694 non-null int64\n",
      "ps_ind_02_cat     21694 non-null int64\n",
      "ps_ind_03         21694 non-null int64\n",
      "ps_ind_04_cat     21694 non-null int64\n",
      "ps_ind_05_cat     21694 non-null int64\n",
      "ps_ind_06_bin     21694 non-null int64\n",
      "ps_ind_07_bin     21694 non-null int64\n",
      "ps_ind_08_bin     21694 non-null int64\n",
      "ps_ind_09_bin     21694 non-null int64\n",
      "ps_ind_10_bin     21694 non-null int64\n",
      "ps_ind_11_bin     21694 non-null int64\n",
      "ps_ind_12_bin     21694 non-null int64\n",
      "ps_ind_13_bin     21694 non-null int64\n",
      "ps_ind_14         21694 non-null int64\n",
      "ps_ind_15         21694 non-null int64\n",
      "ps_ind_16_bin     21694 non-null int64\n",
      "ps_ind_17_bin     21694 non-null int64\n",
      "ps_ind_18_bin     21694 non-null int64\n",
      "ps_reg_01         21694 non-null float64\n",
      "ps_reg_02         21694 non-null float64\n",
      "ps_reg_03         21694 non-null float64\n",
      "ps_car_01_cat     21694 non-null int64\n",
      "ps_car_02_cat     21694 non-null int64\n",
      "ps_car_03_cat     21694 non-null int64\n",
      "ps_car_04_cat     21694 non-null int64\n",
      "ps_car_05_cat     21694 non-null int64\n",
      "ps_car_06_cat     21694 non-null int64\n",
      "ps_car_07_cat     21694 non-null int64\n",
      "ps_car_08_cat     21694 non-null int64\n",
      "ps_car_09_cat     21694 non-null int64\n",
      "ps_car_10_cat     21694 non-null int64\n",
      "ps_car_11_cat     21694 non-null int64\n",
      "ps_car_11         21694 non-null int64\n",
      "ps_car_12         21694 non-null float64\n",
      "ps_car_13         21694 non-null float64\n",
      "ps_car_14         21694 non-null float64\n",
      "ps_car_15         21694 non-null float64\n",
      "ps_calc_01        21694 non-null float64\n",
      "ps_calc_02        21694 non-null float64\n",
      "ps_calc_03        21694 non-null float64\n",
      "ps_calc_04        21694 non-null int64\n",
      "ps_calc_05        21694 non-null int64\n",
      "ps_calc_06        21694 non-null int64\n",
      "ps_calc_07        21694 non-null int64\n",
      "ps_calc_08        21694 non-null int64\n",
      "ps_calc_09        21694 non-null int64\n",
      "ps_calc_10        21694 non-null int64\n",
      "ps_calc_11        21694 non-null int64\n",
      "ps_calc_12        21694 non-null int64\n",
      "ps_calc_13        21694 non-null int64\n",
      "ps_calc_14        21694 non-null int64\n",
      "ps_calc_15_bin    21694 non-null int64\n",
      "ps_calc_16_bin    21694 non-null int64\n",
      "ps_calc_17_bin    21694 non-null int64\n",
      "ps_calc_18_bin    21694 non-null int64\n",
      "ps_calc_19_bin    21694 non-null int64\n",
      "ps_calc_20_bin    21694 non-null int64\n",
      "dtypes: float64(10), int64(49)\n",
      "memory usage: 9.9 MB\n"
     ]
    }
   ],
   "source": [
    "pos_data = data[(data.target == 1)]\n",
    "neg_data = data[(data.target == 0)]\n",
    "pos_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 21694 entries, 564168 to 583460\n",
      "Data columns (total 59 columns):\n",
      "id                21694 non-null int64\n",
      "target            21694 non-null int64\n",
      "ps_ind_01         21694 non-null int64\n",
      "ps_ind_02_cat     21694 non-null int64\n",
      "ps_ind_03         21694 non-null int64\n",
      "ps_ind_04_cat     21694 non-null int64\n",
      "ps_ind_05_cat     21694 non-null int64\n",
      "ps_ind_06_bin     21694 non-null int64\n",
      "ps_ind_07_bin     21694 non-null int64\n",
      "ps_ind_08_bin     21694 non-null int64\n",
      "ps_ind_09_bin     21694 non-null int64\n",
      "ps_ind_10_bin     21694 non-null int64\n",
      "ps_ind_11_bin     21694 non-null int64\n",
      "ps_ind_12_bin     21694 non-null int64\n",
      "ps_ind_13_bin     21694 non-null int64\n",
      "ps_ind_14         21694 non-null int64\n",
      "ps_ind_15         21694 non-null int64\n",
      "ps_ind_16_bin     21694 non-null int64\n",
      "ps_ind_17_bin     21694 non-null int64\n",
      "ps_ind_18_bin     21694 non-null int64\n",
      "ps_reg_01         21694 non-null float64\n",
      "ps_reg_02         21694 non-null float64\n",
      "ps_reg_03         21694 non-null float64\n",
      "ps_car_01_cat     21694 non-null int64\n",
      "ps_car_02_cat     21694 non-null int64\n",
      "ps_car_03_cat     21694 non-null int64\n",
      "ps_car_04_cat     21694 non-null int64\n",
      "ps_car_05_cat     21694 non-null int64\n",
      "ps_car_06_cat     21694 non-null int64\n",
      "ps_car_07_cat     21694 non-null int64\n",
      "ps_car_08_cat     21694 non-null int64\n",
      "ps_car_09_cat     21694 non-null int64\n",
      "ps_car_10_cat     21694 non-null int64\n",
      "ps_car_11_cat     21694 non-null int64\n",
      "ps_car_11         21694 non-null int64\n",
      "ps_car_12         21694 non-null float64\n",
      "ps_car_13         21694 non-null float64\n",
      "ps_car_14         21694 non-null float64\n",
      "ps_car_15         21694 non-null float64\n",
      "ps_calc_01        21694 non-null float64\n",
      "ps_calc_02        21694 non-null float64\n",
      "ps_calc_03        21694 non-null float64\n",
      "ps_calc_04        21694 non-null int64\n",
      "ps_calc_05        21694 non-null int64\n",
      "ps_calc_06        21694 non-null int64\n",
      "ps_calc_07        21694 non-null int64\n",
      "ps_calc_08        21694 non-null int64\n",
      "ps_calc_09        21694 non-null int64\n",
      "ps_calc_10        21694 non-null int64\n",
      "ps_calc_11        21694 non-null int64\n",
      "ps_calc_12        21694 non-null int64\n",
      "ps_calc_13        21694 non-null int64\n",
      "ps_calc_14        21694 non-null int64\n",
      "ps_calc_15_bin    21694 non-null int64\n",
      "ps_calc_16_bin    21694 non-null int64\n",
      "ps_calc_17_bin    21694 non-null int64\n",
      "ps_calc_18_bin    21694 non-null int64\n",
      "ps_calc_19_bin    21694 non-null int64\n",
      "ps_calc_20_bin    21694 non-null int64\n",
      "dtypes: float64(10), int64(49)\n",
      "memory usage: 9.9 MB\n"
     ]
    }
   ],
   "source": [
    "# sample negatives\n",
    "neg_data = neg_data.sample(n=21694)\n",
    "neg_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 43388 entries, 0 to 43387\n",
      "Data columns (total 59 columns):\n",
      "id                43388 non-null int64\n",
      "target            43388 non-null int64\n",
      "ps_ind_01         43388 non-null int64\n",
      "ps_ind_02_cat     43388 non-null int64\n",
      "ps_ind_03         43388 non-null int64\n",
      "ps_ind_04_cat     43388 non-null int64\n",
      "ps_ind_05_cat     43388 non-null int64\n",
      "ps_ind_06_bin     43388 non-null int64\n",
      "ps_ind_07_bin     43388 non-null int64\n",
      "ps_ind_08_bin     43388 non-null int64\n",
      "ps_ind_09_bin     43388 non-null int64\n",
      "ps_ind_10_bin     43388 non-null int64\n",
      "ps_ind_11_bin     43388 non-null int64\n",
      "ps_ind_12_bin     43388 non-null int64\n",
      "ps_ind_13_bin     43388 non-null int64\n",
      "ps_ind_14         43388 non-null int64\n",
      "ps_ind_15         43388 non-null int64\n",
      "ps_ind_16_bin     43388 non-null int64\n",
      "ps_ind_17_bin     43388 non-null int64\n",
      "ps_ind_18_bin     43388 non-null int64\n",
      "ps_reg_01         43388 non-null float64\n",
      "ps_reg_02         43388 non-null float64\n",
      "ps_reg_03         43388 non-null float64\n",
      "ps_car_01_cat     43388 non-null int64\n",
      "ps_car_02_cat     43388 non-null int64\n",
      "ps_car_03_cat     43388 non-null int64\n",
      "ps_car_04_cat     43388 non-null int64\n",
      "ps_car_05_cat     43388 non-null int64\n",
      "ps_car_06_cat     43388 non-null int64\n",
      "ps_car_07_cat     43388 non-null int64\n",
      "ps_car_08_cat     43388 non-null int64\n",
      "ps_car_09_cat     43388 non-null int64\n",
      "ps_car_10_cat     43388 non-null int64\n",
      "ps_car_11_cat     43388 non-null int64\n",
      "ps_car_11         43388 non-null int64\n",
      "ps_car_12         43388 non-null float64\n",
      "ps_car_13         43388 non-null float64\n",
      "ps_car_14         43388 non-null float64\n",
      "ps_car_15         43388 non-null float64\n",
      "ps_calc_01        43388 non-null float64\n",
      "ps_calc_02        43388 non-null float64\n",
      "ps_calc_03        43388 non-null float64\n",
      "ps_calc_04        43388 non-null int64\n",
      "ps_calc_05        43388 non-null int64\n",
      "ps_calc_06        43388 non-null int64\n",
      "ps_calc_07        43388 non-null int64\n",
      "ps_calc_08        43388 non-null int64\n",
      "ps_calc_09        43388 non-null int64\n",
      "ps_calc_10        43388 non-null int64\n",
      "ps_calc_11        43388 non-null int64\n",
      "ps_calc_12        43388 non-null int64\n",
      "ps_calc_13        43388 non-null int64\n",
      "ps_calc_14        43388 non-null int64\n",
      "ps_calc_15_bin    43388 non-null int64\n",
      "ps_calc_16_bin    43388 non-null int64\n",
      "ps_calc_17_bin    43388 non-null int64\n",
      "ps_calc_18_bin    43388 non-null int64\n",
      "ps_calc_19_bin    43388 non-null int64\n",
      "ps_calc_20_bin    43388 non-null int64\n",
      "dtypes: float64(10), int64(49)\n",
      "memory usage: 19.5 MB\n"
     ]
    }
   ],
   "source": [
    "all_data = pd.concat([pos_data, neg_data])\n",
    "all_data = all_data.sample(frac=1).reset_index(drop=True) # shuffle\n",
    "all_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>...</th>\n",
       "      <th>47</th>\n",
       "      <th>48</th>\n",
       "      <th>49</th>\n",
       "      <th>50</th>\n",
       "      <th>51</th>\n",
       "      <th>52</th>\n",
       "      <th>53</th>\n",
       "      <th>54</th>\n",
       "      <th>55</th>\n",
       "      <th>56</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>397909</td>\n",
       "      <td>1</td>\n",
       "      <td>0.006190</td>\n",
       "      <td>-0.540064</td>\n",
       "      <td>-0.176771</td>\n",
       "      <td>1.149286</td>\n",
       "      <td>-0.336169</td>\n",
       "      <td>-0.735220</td>\n",
       "      <td>1.537339</td>\n",
       "      <td>-0.462444</td>\n",
       "      <td>...</td>\n",
       "      <td>1.095924</td>\n",
       "      <td>-0.361753</td>\n",
       "      <td>-0.511807</td>\n",
       "      <td>-0.564091</td>\n",
       "      <td>-0.375711</td>\n",
       "      <td>0.770888</td>\n",
       "      <td>0.895072</td>\n",
       "      <td>-0.632721</td>\n",
       "      <td>1.376676</td>\n",
       "      <td>-0.421670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>993433</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.491145</td>\n",
       "      <td>-0.540064</td>\n",
       "      <td>1.643982</td>\n",
       "      <td>-0.864413</td>\n",
       "      <td>0.324924</td>\n",
       "      <td>1.360137</td>\n",
       "      <td>-0.650475</td>\n",
       "      <td>-0.462444</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.042133</td>\n",
       "      <td>0.470427</td>\n",
       "      <td>0.666527</td>\n",
       "      <td>0.524612</td>\n",
       "      <td>-0.375711</td>\n",
       "      <td>0.770888</td>\n",
       "      <td>-1.117228</td>\n",
       "      <td>-0.632721</td>\n",
       "      <td>1.376676</td>\n",
       "      <td>-0.421670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>804416</td>\n",
       "      <td>1</td>\n",
       "      <td>1.498193</td>\n",
       "      <td>3.892554</td>\n",
       "      <td>2.372283</td>\n",
       "      <td>1.149286</td>\n",
       "      <td>-0.336169</td>\n",
       "      <td>-0.735220</td>\n",
       "      <td>1.537339</td>\n",
       "      <td>-0.462444</td>\n",
       "      <td>...</td>\n",
       "      <td>1.095924</td>\n",
       "      <td>-0.361753</td>\n",
       "      <td>1.844861</td>\n",
       "      <td>2.339118</td>\n",
       "      <td>-0.375711</td>\n",
       "      <td>0.770888</td>\n",
       "      <td>0.895072</td>\n",
       "      <td>-0.632721</td>\n",
       "      <td>1.376676</td>\n",
       "      <td>-0.421670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>576857</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.491145</td>\n",
       "      <td>0.937475</td>\n",
       "      <td>-0.176771</td>\n",
       "      <td>1.149286</td>\n",
       "      <td>-0.336169</td>\n",
       "      <td>-0.735220</td>\n",
       "      <td>-0.650475</td>\n",
       "      <td>2.162426</td>\n",
       "      <td>...</td>\n",
       "      <td>0.668313</td>\n",
       "      <td>2.134787</td>\n",
       "      <td>0.666527</td>\n",
       "      <td>0.524612</td>\n",
       "      <td>-0.375711</td>\n",
       "      <td>-1.297206</td>\n",
       "      <td>-1.117228</td>\n",
       "      <td>-0.632721</td>\n",
       "      <td>1.376676</td>\n",
       "      <td>-0.421670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>267011</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.988479</td>\n",
       "      <td>0.937475</td>\n",
       "      <td>0.187380</td>\n",
       "      <td>1.149286</td>\n",
       "      <td>-0.336169</td>\n",
       "      <td>-0.735220</td>\n",
       "      <td>1.537339</td>\n",
       "      <td>-0.462444</td>\n",
       "      <td>...</td>\n",
       "      <td>0.668313</td>\n",
       "      <td>2.134787</td>\n",
       "      <td>0.077360</td>\n",
       "      <td>0.161711</td>\n",
       "      <td>-0.375711</td>\n",
       "      <td>0.770888</td>\n",
       "      <td>-1.117228</td>\n",
       "      <td>-0.632721</td>\n",
       "      <td>-0.726388</td>\n",
       "      <td>2.371523</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 59 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  target         0         1         2         3         4         5  \\\n",
       "0  397909       1  0.006190 -0.540064 -0.176771  1.149286 -0.336169 -0.735220   \n",
       "1  993433       0 -0.491145 -0.540064  1.643982 -0.864413  0.324924  1.360137   \n",
       "2  804416       1  1.498193  3.892554  2.372283  1.149286 -0.336169 -0.735220   \n",
       "3  576857       0 -0.491145  0.937475 -0.176771  1.149286 -0.336169 -0.735220   \n",
       "4  267011       0 -0.988479  0.937475  0.187380  1.149286 -0.336169 -0.735220   \n",
       "\n",
       "          6         7    ...           47        48        49        50  \\\n",
       "0  1.537339 -0.462444    ...     1.095924 -0.361753 -0.511807 -0.564091   \n",
       "1 -0.650475 -0.462444    ...    -1.042133  0.470427  0.666527  0.524612   \n",
       "2  1.537339 -0.462444    ...     1.095924 -0.361753  1.844861  2.339118   \n",
       "3 -0.650475  2.162426    ...     0.668313  2.134787  0.666527  0.524612   \n",
       "4  1.537339 -0.462444    ...     0.668313  2.134787  0.077360  0.161711   \n",
       "\n",
       "         51        52        53        54        55        56  \n",
       "0 -0.375711  0.770888  0.895072 -0.632721  1.376676 -0.421670  \n",
       "1 -0.375711  0.770888 -1.117228 -0.632721  1.376676 -0.421670  \n",
       "2 -0.375711  0.770888  0.895072 -0.632721  1.376676 -0.421670  \n",
       "3 -0.375711 -1.297206 -1.117228 -0.632721  1.376676 -0.421670  \n",
       "4 -0.375711  0.770888 -1.117228 -0.632721 -0.726388  2.371523  \n",
       "\n",
       "[5 rows x 59 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# standardize data\n",
    "df_x = all_data.iloc[:,2:]\n",
    "df_y = all_data.iloc[:,:2]\n",
    "\n",
    "scaler = StandardScaler().fit(df_x)\n",
    "df_x = pd.DataFrame(scaler.transform(df_x))\n",
    "\n",
    "all_data = pd.concat([df_y, df_x], axis=1)\n",
    "all_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40000    0\n",
       "40001    1\n",
       "40002    0\n",
       "40003    1\n",
       "40004    1\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train = all_data.iloc[:40000,2:]\n",
    "y_train = all_data.iloc[:40000,1]\n",
    "x_test = all_data.iloc[40000:,2:]\n",
    "y_test = all_data.iloc[40000:,1]\n",
    "y_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MLPClassifier(activation='logistic', alpha=0.0001, batch_size='auto',\n",
       "       beta_1=0.9, beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
       "       hidden_layer_sizes=300, learning_rate='constant',\n",
       "       learning_rate_init=0.001, max_iter=200, momentum=0.9,\n",
       "       nesterovs_momentum=True, power_t=0.5, random_state=None,\n",
       "       shuffle=True, solver='adam', tol=0.0001, validation_fraction=0.1,\n",
       "       verbose=False, warm_start=False)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp = MLPClassifier(hidden_layer_sizes=(300), \n",
    "                    activation='logistic')\n",
    "\n",
    "mlp.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5832349468713105"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp.score(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>ps_ind_01</th>\n",
       "      <th>ps_ind_02_cat</th>\n",
       "      <th>ps_ind_03</th>\n",
       "      <th>ps_ind_04_cat</th>\n",
       "      <th>ps_ind_05_cat</th>\n",
       "      <th>ps_ind_06_bin</th>\n",
       "      <th>ps_ind_07_bin</th>\n",
       "      <th>ps_ind_08_bin</th>\n",
       "      <th>ps_ind_09_bin</th>\n",
       "      <th>...</th>\n",
       "      <th>ps_calc_11</th>\n",
       "      <th>ps_calc_12</th>\n",
       "      <th>ps_calc_13</th>\n",
       "      <th>ps_calc_14</th>\n",
       "      <th>ps_calc_15_bin</th>\n",
       "      <th>ps_calc_16_bin</th>\n",
       "      <th>ps_calc_17_bin</th>\n",
       "      <th>ps_calc_18_bin</th>\n",
       "      <th>ps_calc_19_bin</th>\n",
       "      <th>ps_calc_20_bin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 58 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  ps_ind_01  ps_ind_02_cat  ps_ind_03  ps_ind_04_cat  ps_ind_05_cat  \\\n",
       "0   0          0              1          8              1              0   \n",
       "1   1          4              2          5              1              0   \n",
       "2   2          5              1          3              0              0   \n",
       "3   3          0              1          6              0              0   \n",
       "4   4          5              1          7              0              0   \n",
       "\n",
       "   ps_ind_06_bin  ps_ind_07_bin  ps_ind_08_bin  ps_ind_09_bin       ...        \\\n",
       "0              0              1              0              0       ...         \n",
       "1              0              0              0              1       ...         \n",
       "2              0              0              0              1       ...         \n",
       "3              1              0              0              0       ...         \n",
       "4              0              0              0              1       ...         \n",
       "\n",
       "   ps_calc_11  ps_calc_12  ps_calc_13  ps_calc_14  ps_calc_15_bin  \\\n",
       "0           1           1           1          12               0   \n",
       "1           2           0           3          10               0   \n",
       "2           4           0           2           4               0   \n",
       "3           5           1           0           5               1   \n",
       "4           4           0           0           4               0   \n",
       "\n",
       "   ps_calc_16_bin  ps_calc_17_bin  ps_calc_18_bin  ps_calc_19_bin  \\\n",
       "0               1               1               0               0   \n",
       "1               0               1               1               0   \n",
       "2               0               0               0               0   \n",
       "3               0               1               0               0   \n",
       "4               1               1               0               0   \n",
       "\n",
       "   ps_calc_20_bin  \n",
       "0               1  \n",
       "1               1  \n",
       "2               0  \n",
       "3               0  \n",
       "4               1  \n",
       "\n",
       "[5 rows x 58 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data = pd.read_csv('../kaggle_porto_seguro/test.csv')\n",
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = mlp.predict(test_data.iloc[:,1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  target\n",
       "0   0       0\n",
       "1   1       1\n",
       "2   2       0\n",
       "3   3       0\n",
       "4   4       1"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission = pd.DataFrame()\n",
    "submission['id'] = test_data.iloc[:, 0]\n",
    "submission['target'] = predictions\n",
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submission.to_csv('kaggle_submission_nn.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
